{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d70555bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf318244",
   "metadata": {},
   "source": [
    "# 原始数据分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "158a7d7838005f66",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T06:53:35.470785Z",
     "start_time": "2025-03-19T06:53:14.412478Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18103954"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_path = os.path.join(os.path.expanduser('~'), 'Data/tick10_16_partition_by_symbol')\n",
    "# base_path = os.path.join(os.path.expanduser('~'), 'Data/tick10_test_1_day')\n",
    "df = pd.concat((pd.read_parquet(os.path.join(base_path, f)) for f in os.listdir(base_path) if f.endswith('.parquet')), ignore_index=True)\n",
    "df['date_time'] = pd.to_datetime(df['date_time'])\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e24843511e2439b6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T06:55:49.614409Z",
     "start_time": "2025-03-19T06:55:49.240157Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "symbol\n",
       "600050    1162059\n",
       "600150    1162141\n",
       "600276    1161886\n",
       "600309    1157617\n",
       "600519    1154757\n",
       "600668     719083\n",
       "600690    1158402\n",
       "600900    1160960\n",
       "601088    1156982\n",
       "601138    1161342\n",
       "601288    1162509\n",
       "601398    1162213\n",
       "601628    1151152\n",
       "601766    1156753\n",
       "601816    1154091\n",
       "601857    1162007\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp =  df.groupby('symbol').size()\n",
    "tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af1f6606d7918052",
   "metadata": {},
   "source": [
    "# 特征因子结果分析"
   ]
  },
  {
   "cell_type": "code",
   "id": "d0611880",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T07:14:58.945849Z",
     "start_time": "2025-03-19T07:14:58.783800Z"
    }
   },
   "source": [
    "base_path = os.path.join(os.path.expanduser('~'), 'Data/tick10_16_partition_by_symbol_feature/')\n",
    "one_feature_file = os.path.join(base_path, \"agg_feature_sh600050.parquet\")\n",
    "\n",
    "feature_df = pd.read_parquet(one_feature_file)\n",
    "feature_df"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       symbol_first  date_time_count  wap_balance_sum  wap_balance_mean  \\\n",
       "0          600050.0              201     9.187673e+07      4.570981e+05   \n",
       "1          600050.0              200     1.442762e+08      7.213810e+05   \n",
       "2          600050.0              200     1.622664e+08      8.113321e+05   \n",
       "3          600050.0              200     2.324766e+08      1.162383e+06   \n",
       "4          600050.0              200     1.859113e+08      9.295566e+05   \n",
       "...             ...              ...              ...               ...   \n",
       "51869      600050.0              200     4.264304e+08      2.132152e+06   \n",
       "51870      600050.0              200     5.517906e+08      2.758953e+06   \n",
       "51871      600050.0              200     4.961578e+08      2.480789e+06   \n",
       "51872      600050.0              200     1.774789e+08      1.258715e+06   \n",
       "51873      600050.0                1     0.000000e+00               NaN   \n",
       "\n",
       "       wap_balance_std  price_spread_sum  price_spread_mean  price_spread_std  \\\n",
       "0         2.305609e+05          0.453194           0.002255          0.000269   \n",
       "1         2.974910e+05          0.445268           0.002226          0.000156   \n",
       "2         3.636966e+05          0.442353           0.002212          0.000218   \n",
       "3         5.652437e+05          0.435883           0.002179          0.000004   \n",
       "4         3.818619e+05          0.431117           0.002156          0.000009   \n",
       "...                ...               ...                ...               ...   \n",
       "51869     3.041869e+05          0.455301           0.002277          0.000002   \n",
       "51870     4.704807e+05          0.455701           0.002279          0.000003   \n",
       "51871     1.136534e+06          0.455540           0.002278          0.000003   \n",
       "51872     2.370655e+05          0.321551           0.001608          0.001043   \n",
       "51873              NaN          0.000000           0.000000               NaN   \n",
       "\n",
       "       bid_spread_sum  bid_spread_mean  ...  log_return_bid7_std  \\\n",
       "0       -7.630346e+07    -3.796192e+05  ...             0.355133   \n",
       "1       -1.186517e+08    -5.932585e+05  ...             0.140523   \n",
       "2       -1.129058e+08    -5.645290e+05  ...             0.191519   \n",
       "3       -1.500403e+08    -7.502014e+05  ...             0.112825   \n",
       "4       -1.537648e+08    -7.688239e+05  ...             0.167041   \n",
       "...               ...              ...  ...                  ...   \n",
       "51869   -5.035514e+08    -2.517757e+06  ...             0.052200   \n",
       "51870   -6.690131e+08    -3.345066e+06  ...             0.116537   \n",
       "51871   -1.947891e+08    -9.739456e+05  ...             0.055880   \n",
       "51872   -1.471629e+08    -7.358145e+05  ...             0.006366   \n",
       "51873   -1.438199e+06    -1.438199e+06  ...                  NaN   \n",
       "\n",
       "       log_return_bid7_realized_volatility  log_return_bid8_sum  \\\n",
       "0                                 5.012214             0.011198   \n",
       "1                                 1.982512             0.002225   \n",
       "2                                 2.701775             0.008850   \n",
       "3                                 1.591624             0.002200   \n",
       "4                                 2.356415             0.013101   \n",
       "...                                    ...                  ...   \n",
       "51869                             0.736373             0.000000   \n",
       "51870                             1.644003             0.000000   \n",
       "51871                             0.789051            -0.002301   \n",
       "51872                             0.077565             0.000000   \n",
       "51873                             0.000000             0.000000   \n",
       "\n",
       "       log_return_bid8_mean  log_return_bid8_std  \\\n",
       "0                  0.000056             0.001041   \n",
       "1                  0.000011             0.000792   \n",
       "2                  0.000044             0.000767   \n",
       "3                  0.000011             0.000604   \n",
       "4                  0.000066             0.000720   \n",
       "...                     ...                  ...   \n",
       "51869              0.000000             0.000326   \n",
       "51870              0.000000             0.000565   \n",
       "51871             -0.000012             0.000365   \n",
       "51872              0.000000             0.000000   \n",
       "51873                   NaN                  NaN   \n",
       "\n",
       "       log_return_bid8_realized_volatility  log_return_bid9_sum  \\\n",
       "0                                 0.014706             1.944891   \n",
       "1                                 0.011169             0.668990   \n",
       "2                                 0.010841            -1.270580   \n",
       "3                                 0.008521             1.055113   \n",
       "4                                 0.010200            -0.005068   \n",
       "...                                    ...                  ...   \n",
       "51869                             0.004603             0.008056   \n",
       "51870                             0.007973            -0.095601   \n",
       "51871                             0.005151             0.148828   \n",
       "51872                             0.000000             0.012050   \n",
       "51873                             0.000000             0.000000   \n",
       "\n",
       "       log_return_bid9_mean  log_return_bid9_std  \\\n",
       "0                  0.009724             0.353518   \n",
       "1                  0.003345             0.167825   \n",
       "2                 -0.006353             0.227121   \n",
       "3                  0.005276             0.110378   \n",
       "4                 -0.000025             0.211114   \n",
       "...                     ...                  ...   \n",
       "51869              0.000040             0.051375   \n",
       "51870             -0.000478             0.070015   \n",
       "51871              0.000744             0.090811   \n",
       "51872              0.000085             0.007456   \n",
       "51873                   NaN                  NaN   \n",
       "\n",
       "       log_return_bid9_realized_volatility  \n",
       "0                                 4.988876  \n",
       "1                                 2.367940  \n",
       "2                                 3.205195  \n",
       "3                                 1.558860  \n",
       "4                                 2.978136  \n",
       "...                                    ...  \n",
       "51869                             0.724732  \n",
       "51870                             0.987707  \n",
       "51871                             1.281094  \n",
       "51872                             0.088227  \n",
       "51873                             0.000000  \n",
       "\n",
       "[51874 rows x 170 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>symbol_first</th>\n",
       "      <th>date_time_count</th>\n",
       "      <th>wap_balance_sum</th>\n",
       "      <th>wap_balance_mean</th>\n",
       "      <th>wap_balance_std</th>\n",
       "      <th>price_spread_sum</th>\n",
       "      <th>price_spread_mean</th>\n",
       "      <th>price_spread_std</th>\n",
       "      <th>bid_spread_sum</th>\n",
       "      <th>bid_spread_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>log_return_bid7_std</th>\n",
       "      <th>log_return_bid7_realized_volatility</th>\n",
       "      <th>log_return_bid8_sum</th>\n",
       "      <th>log_return_bid8_mean</th>\n",
       "      <th>log_return_bid8_std</th>\n",
       "      <th>log_return_bid8_realized_volatility</th>\n",
       "      <th>log_return_bid9_sum</th>\n",
       "      <th>log_return_bid9_mean</th>\n",
       "      <th>log_return_bid9_std</th>\n",
       "      <th>log_return_bid9_realized_volatility</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>201</td>\n",
       "      <td>9.187673e+07</td>\n",
       "      <td>4.570981e+05</td>\n",
       "      <td>2.305609e+05</td>\n",
       "      <td>0.453194</td>\n",
       "      <td>0.002255</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>-7.630346e+07</td>\n",
       "      <td>-3.796192e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.355133</td>\n",
       "      <td>5.012214</td>\n",
       "      <td>0.011198</td>\n",
       "      <td>0.000056</td>\n",
       "      <td>0.001041</td>\n",
       "      <td>0.014706</td>\n",
       "      <td>1.944891</td>\n",
       "      <td>0.009724</td>\n",
       "      <td>0.353518</td>\n",
       "      <td>4.988876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>1.442762e+08</td>\n",
       "      <td>7.213810e+05</td>\n",
       "      <td>2.974910e+05</td>\n",
       "      <td>0.445268</td>\n",
       "      <td>0.002226</td>\n",
       "      <td>0.000156</td>\n",
       "      <td>-1.186517e+08</td>\n",
       "      <td>-5.932585e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.140523</td>\n",
       "      <td>1.982512</td>\n",
       "      <td>0.002225</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000792</td>\n",
       "      <td>0.011169</td>\n",
       "      <td>0.668990</td>\n",
       "      <td>0.003345</td>\n",
       "      <td>0.167825</td>\n",
       "      <td>2.367940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>1.622664e+08</td>\n",
       "      <td>8.113321e+05</td>\n",
       "      <td>3.636966e+05</td>\n",
       "      <td>0.442353</td>\n",
       "      <td>0.002212</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>-1.129058e+08</td>\n",
       "      <td>-5.645290e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.191519</td>\n",
       "      <td>2.701775</td>\n",
       "      <td>0.008850</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.000767</td>\n",
       "      <td>0.010841</td>\n",
       "      <td>-1.270580</td>\n",
       "      <td>-0.006353</td>\n",
       "      <td>0.227121</td>\n",
       "      <td>3.205195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>2.324766e+08</td>\n",
       "      <td>1.162383e+06</td>\n",
       "      <td>5.652437e+05</td>\n",
       "      <td>0.435883</td>\n",
       "      <td>0.002179</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>-1.500403e+08</td>\n",
       "      <td>-7.502014e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.112825</td>\n",
       "      <td>1.591624</td>\n",
       "      <td>0.002200</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000604</td>\n",
       "      <td>0.008521</td>\n",
       "      <td>1.055113</td>\n",
       "      <td>0.005276</td>\n",
       "      <td>0.110378</td>\n",
       "      <td>1.558860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>1.859113e+08</td>\n",
       "      <td>9.295566e+05</td>\n",
       "      <td>3.818619e+05</td>\n",
       "      <td>0.431117</td>\n",
       "      <td>0.002156</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>-1.537648e+08</td>\n",
       "      <td>-7.688239e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.167041</td>\n",
       "      <td>2.356415</td>\n",
       "      <td>0.013101</td>\n",
       "      <td>0.000066</td>\n",
       "      <td>0.000720</td>\n",
       "      <td>0.010200</td>\n",
       "      <td>-0.005068</td>\n",
       "      <td>-0.000025</td>\n",
       "      <td>0.211114</td>\n",
       "      <td>2.978136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51869</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>4.264304e+08</td>\n",
       "      <td>2.132152e+06</td>\n",
       "      <td>3.041869e+05</td>\n",
       "      <td>0.455301</td>\n",
       "      <td>0.002277</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>-5.035514e+08</td>\n",
       "      <td>-2.517757e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.052200</td>\n",
       "      <td>0.736373</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000326</td>\n",
       "      <td>0.004603</td>\n",
       "      <td>0.008056</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>0.051375</td>\n",
       "      <td>0.724732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51870</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>5.517906e+08</td>\n",
       "      <td>2.758953e+06</td>\n",
       "      <td>4.704807e+05</td>\n",
       "      <td>0.455701</td>\n",
       "      <td>0.002279</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-6.690131e+08</td>\n",
       "      <td>-3.345066e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.116537</td>\n",
       "      <td>1.644003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000565</td>\n",
       "      <td>0.007973</td>\n",
       "      <td>-0.095601</td>\n",
       "      <td>-0.000478</td>\n",
       "      <td>0.070015</td>\n",
       "      <td>0.987707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51871</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>4.961578e+08</td>\n",
       "      <td>2.480789e+06</td>\n",
       "      <td>1.136534e+06</td>\n",
       "      <td>0.455540</td>\n",
       "      <td>0.002278</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-1.947891e+08</td>\n",
       "      <td>-9.739456e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055880</td>\n",
       "      <td>0.789051</td>\n",
       "      <td>-0.002301</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>0.000365</td>\n",
       "      <td>0.005151</td>\n",
       "      <td>0.148828</td>\n",
       "      <td>0.000744</td>\n",
       "      <td>0.090811</td>\n",
       "      <td>1.281094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51872</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>200</td>\n",
       "      <td>1.774789e+08</td>\n",
       "      <td>1.258715e+06</td>\n",
       "      <td>2.370655e+05</td>\n",
       "      <td>0.321551</td>\n",
       "      <td>0.001608</td>\n",
       "      <td>0.001043</td>\n",
       "      <td>-1.471629e+08</td>\n",
       "      <td>-7.358145e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006366</td>\n",
       "      <td>0.077565</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.012050</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.007456</td>\n",
       "      <td>0.088227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51873</th>\n",
       "      <td>600050.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.438199e+06</td>\n",
       "      <td>-1.438199e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51874 rows × 170 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T07:15:50.861482Z",
     "start_time": "2025-03-19T07:15:50.857150Z"
    }
   },
   "cell_type": "code",
   "source": "feature_df.columns",
   "id": "24883c404e282f07",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['symbol_first', 'date_time_count', 'wap_balance_sum',\n",
       "       'wap_balance_mean', 'wap_balance_std', 'price_spread_sum',\n",
       "       'price_spread_mean', 'price_spread_std', 'bid_spread_sum',\n",
       "       'bid_spread_mean',\n",
       "       ...\n",
       "       'log_return_bid7_std', 'log_return_bid7_realized_volatility',\n",
       "       'log_return_bid8_sum', 'log_return_bid8_mean', 'log_return_bid8_std',\n",
       "       'log_return_bid8_realized_volatility', 'log_return_bid9_sum',\n",
       "       'log_return_bid9_mean', 'log_return_bid9_std',\n",
       "       'log_return_bid9_realized_volatility'],\n",
       "      dtype='object', length=170)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "d1d3266ab18881f2"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
